\documentclass{article}

\usepackage[paperwidth=19cm, paperheight=10.8cm,top=0.1cm,bottom=0.1cm,left=0.2cm,right=0.2cm]{geometry}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{amsmath}
\pagenumbering{gobble}
\begin{document}
\begin{algorithm}
    \caption{\textbf{D}eep \textbf{Q}-\textbf{N}etwork (Timestep Based)}
    \begin{algorithmic}[1]
        \STATE Randomly initialize action-value network $Q_\phi$ with parameter $\phi\gets \phi_0$.
        \STATE Initialize target network $Q_{\phi^\prime}$ via parameter copy: $\phi^\prime\gets\phi$.
        \STATE Initialize Replay Buffer $\mathcal{B}$, and collect some transitions $\{s,a,r,s^\prime,d\}$ by take random actions before training starts.
        \FOR{$t=1,2,\ldots,T$}
        \STATE Take an action $a_t$ sampled from $\epsilon-greedy$ policy:
        $$
            a_t=\left\{
            \begin{aligned}
                 & random\ from\ \mathcal{A},\quad \mathrm{if}\ x< \epsilon           \\
                 & \underset{a\in\mathcal{A}}{\arg\max}\ Q(s_t,a),\quad \mathrm{else}
            \end{aligned}
            \right.
        $$

        \STATE Observe $\{r_{t+1},s_{t+1}\}$ and store transition $\{s_t,a_t,r_{t+1},s_{t+1},d_{t+1}\}$ in $\mathcal{B}$. If episode ends, reset environment.
        \STATE Randomly sample a minibatch $B$ with transitions $\{s_i,a_i,r_i,s_i^\prime,d_i\}_{i=1,2,\ldots,|B|}$ from $\mathcal{B}$.
        \STATE Compute TD error:
        $$
            \begin{aligned}
                y_i&=r_i+(1-d_i)\cdot \gamma\ \underset{a^\prime\in \mathcal{A}}{\max}\ Q_{\phi^\prime}(s_i^\prime,a^\prime)\\
                L_{Q}&=\frac{1}{|B|}\sum_{i=1}^{|B|}\left( y_i-Q_\phi(s_i,a_i) \right)^2
            \end{aligned}
        $$
        \STATE Update $Q$-Network parameter $\phi$ via any gradient \textbf{descent} algorithm:
        $$
            \phi \gets \phi + \nabla_\phi L_{Q}
        $$

        \STATE Update target network parameter if $\mathcal{D}$ steps pass: $\phi^\prime\gets \phi$.

        \ENDFOR




    \end{algorithmic}
\end{algorithm}
\end{document}
